{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 线性规划"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn\n",
    "import torch.optim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用 lstsq() 最小二乘"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[  4.6667],\n",
       "        [  2.6667],\n",
       "        [-12.0000]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor([[1., 1., 1.], [2., 3., 1.],\n",
    "        [3., 5., 1.], [4., 2., 1.], [5., 4., 1.]])\n",
    "y = torch.tensor([-10., 12., 14., 16., 18.])\n",
    "wr, _ = torch.lstsq(y, x)\n",
    "w = wr[:3]\n",
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[  4.6667,   3.3333],\n",
       "        [  2.6667,   1.3333],\n",
       "        [-12.0000,  -3.0000]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.tensor([[1., 1., 1.], [2., 3., 1.], \n",
    "        [3., 5., 1.], [4., 2., 1.], [5., 4., 1.]])\n",
    "y = torch.tensor([[-10., -3.], [12., 14.], [14., 12.], [16., 16.], [18., 16.]])\n",
    "wr, _ = torch.lstsq(y, x)\n",
    "w = wr[:3, :]\n",
    "w"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MSE 损失"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3., grad_fn=<MseLossBackward>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "criterion = torch.nn.MSELoss()\n",
    "pred = torch.arange(5, dtype=torch.float, requires_grad=True)\n",
    "y = torch.ones(5)\n",
    "loss = criterion(pred, y)\n",
    "loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "用优化引擎求解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step = 0, loss = 204, W = [0.0, 0.0, 0.0]\n",
      "step = 1000, loss = 86.8401, W = [0.863132655620575, 0.8542488813400269, 0.8209308981895447]\n",
      "step = 2000, loss = 53.6289, W = [1.443850040435791, 1.3946014642715454, 1.1897931098937988]\n",
      "step = 3000, loss = 47.505, W = [1.7840828895568848, 1.634355902671814, 0.9572443962097168]\n",
      "step = 4000, loss = 44.2888, W = [2.0315651893615723, 1.7021595239639282, 0.2563045024871826]\n",
      "step = 5000, loss = 40.8731, W = [2.3051974773406982, 1.7125362157821655, -0.6180321574211121]\n",
      "step = 6000, loss = 37.5372, W = [2.6193673610687256, 1.6853389739990234, -1.5441280603408813]\n",
      "step = 7000, loss = 34.5258, W = [2.9509589672088623, 1.6359083652496338, -2.4787139892578125]\n",
      "step = 8000, loss = 31.9455, W = [3.259428024291992, 1.602895975112915, -3.4061663150787354]\n",
      "step = 9000, loss = 29.7652, W = [3.5038466453552246, 1.6267775297164917, -4.3211588859558105]\n",
      "step = 10000, loss = 27.9001, W = [3.6783804893493652, 1.7130744457244873, -5.2205023765563965]\n",
      "step = 11000, loss = 26.3032, W = [3.8159337043762207, 1.8293808698654175, -6.101877689361572]\n",
      "step = 12000, loss = 24.9573, W = [3.942659854888916, 1.9499529600143433, -6.963371276855469]\n",
      "step = 13000, loss = 23.8511, W = [4.0643534660339355, 2.068524122238159, -7.801889419555664]\n",
      "step = 14000, loss = 22.9731, W = [4.181187152862549, 2.183582305908203, -8.612066268920898]\n",
      "step = 15000, loss = 22.31, W = [4.292291641235352, 2.293663501739502, -9.385353088378906]\n",
      "step = 16000, loss = 21.8445, W = [4.395956039428711, 2.396719217300415, -10.108356475830078]\n",
      "step = 17000, loss = 21.553, W = [4.489279270172119, 2.489685535430908, -10.760062217712402]\n",
      "step = 18000, loss = 21.4017, W = [4.567684173583984, 2.5678787231445312, -11.30797290802002]\n",
      "step = 19000, loss = 21.3455, W = [4.624871253967285, 2.6249477863311768, -11.707765579223633]\n",
      "step = 20000, loss = 21.3341, W = [4.655961990356445, 2.65598201751709, -11.925153732299805]\n",
      "step = 21000, loss = 21.3333, W = [4.665530681610107, 2.6655328273773193, -11.992058753967285]\n",
      "step = 22000, loss = 21.3333, W = [4.666637897491455, 2.666637659072876, -11.999797821044922]\n",
      "step = 23000, loss = 21.3333, W = [4.666656970977783, 2.666656494140625, -11.999942779541016]\n",
      "step = 24000, loss = 21.3333, W = [4.666667938232422, 2.66666841506958, -12.000000953674316]\n",
      "step = 25000, loss = 21.3333, W = [4.666663646697998, 2.666663885116577, -11.999999046325684]\n",
      "step = 26000, loss = 21.3333, W = [4.666727542877197, 2.6667284965515137, -11.999998092651367]\n",
      "step = 27000, loss = 21.3333, W = [4.666664123535156, 2.6666646003723145, -12.000000953674316]\n",
      "step = 28000, loss = 21.3333, W = [4.666672706604004, 2.6666712760925293, -12.000000953674316]\n",
      "step = 29000, loss = 21.3333, W = [4.666668891906738, 2.6666698455810547, -12.0]\n",
      "step = 30000, loss = 21.3333, W = [4.6666765213012695, 2.6666765213012695, -11.999996185302734]\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1., 1., 1.], [2., 3., 1.], \n",
    "        [3., 5., 1.], [4., 2., 1.], [5., 4., 1.]])\n",
    "y = torch.tensor([-10., 12., 14., 16., 18.])\n",
    "w = torch.zeros(3, requires_grad=True)\n",
    "\n",
    "criterion = torch.nn.MSELoss()\n",
    "optimizer = torch.optim.Adam([w,],)\n",
    "\n",
    "for step in range(30001):\n",
    "    if step:\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    \n",
    "    pred = torch.mv(x, w)\n",
    "    loss = criterion(pred, y)\n",
    "    if step % 1000 == 0:\n",
    "        print('step = {}, loss = {:g}, W = {}'.format(\n",
    "                step, loss, w.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step = 0, loss = 241.184, weights = [0.3011220097541809, -0.6333320140838623], bias=-0.4551149904727936\n",
      "step = 1000, loss = 103.052, weights = [1.1779792308807373, 0.2398810237646103], bias=0.3923540711402893\n",
      "step = 2000, loss = 56.3205, weights = [1.7984486818313599, 0.8443279266357422], bias=0.8723846673965454\n",
      "step = 3000, loss = 46.4747, weights = [2.1704301834106445, 1.1754837036132812], bias=0.8494079113006592\n",
      "step = 4000, loss = 43.5579, weights = [2.387246608734131, 1.3138198852539062], bias=0.312853068113327\n",
      "step = 5000, loss = 40.6367, weights = [2.5850045680999756, 1.3900961875915527], bias=-0.4890872836112976\n",
      "step = 6000, loss = 37.6488, weights = [2.808905601501465, 1.4479899406433105], bias=-1.3870124816894531\n",
      "step = 7000, loss = 34.8404, weights = [3.0497021675109863, 1.48907470703125], bias=-2.3139519691467285\n",
      "step = 8000, loss = 32.3265, weights = [3.2864911556243896, 1.5278198719024658], bias=-3.2405009269714355\n",
      "step = 9000, loss = 30.1285, weights = [3.493490695953369, 1.5891287326812744], bias=-4.156043529510498\n",
      "step = 10000, loss = 28.2209, weights = [3.6584105491638184, 1.6855701208114624], bias=-5.056815147399902\n",
      "step = 11000, loss = 26.5784, weights = [3.7949984073638916, 1.8038238286972046], bias=-5.940746784210205\n",
      "step = 12000, loss = 25.1876, weights = [3.9214067459106445, 1.925907850265503], bias=-6.80579137802124\n",
      "step = 13000, loss = 24.038, weights = [4.043209075927734, 2.0457732677459717], bias=-7.648836612701416\n",
      "step = 14000, loss = 23.1187, weights = [4.160564422607422, 2.162036180496216], bias=-8.464818000793457\n",
      "step = 15000, loss = 22.417, weights = [4.272592544555664, 2.2734322547912598], bias=-9.245805740356445\n",
      "step = 16000, loss = 21.9166, weights = [4.3776702880859375, 2.3781399726867676], bias=-9.979450225830078\n",
      "step = 17000, loss = 21.5951, weights = [4.473110675811768, 2.473362684249878], bias=-10.646408081054688\n",
      "step = 18000, loss = 21.421, weights = [4.554655075073242, 2.554779529571533], bias=-11.216560363769531\n",
      "step = 19000, loss = 21.3511, weights = [4.616230010986328, 2.6162800788879395], bias=-11.647202491760254\n",
      "step = 20000, loss = 21.3348, weights = [4.652169704437256, 2.6521830558776855], bias=-11.898591995239258\n",
      "step = 21000, loss = 21.3334, weights = [4.6647868156433105, 2.664788246154785], bias=-11.986849784851074\n",
      "step = 22000, loss = 21.3333, weights = [4.6666035652160645, 2.6666033267974854], bias=-11.999558448791504\n",
      "step = 23000, loss = 21.3333, weights = [4.666656970977783, 2.666656732559204], bias=-11.999935150146484\n",
      "step = 24000, loss = 21.3333, weights = [4.666679382324219, 2.666679620742798], bias=-11.99999713897705\n",
      "step = 25000, loss = 21.3333, weights = [4.666661262512207, 2.6666595935821533], bias=-12.0\n",
      "step = 26000, loss = 21.3333, weights = [4.666669845581055, 2.6666698455810547], bias=-12.0\n",
      "step = 27000, loss = 21.3333, weights = [4.666667938232422, 2.666668653488159], bias=-12.000000953674316\n",
      "step = 28000, loss = 21.3333, weights = [4.666681289672852, 2.6666812896728516], bias=-11.999998092651367\n",
      "step = 29000, loss = 21.3333, weights = [4.666689395904541, 2.666689395904541], bias=-11.999995231628418\n",
      "step = 30000, loss = 21.3333, weights = [4.666656970977783, 2.666656732559204], bias=-12.00000286102295\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1., 1.], [2., 3.], [3., 5.], [4., 2.], [5., 4.]])\n",
    "y = torch.tensor([-10., 12., 14., 16., 18.]).reshape(-1, 1)\n",
    "\n",
    "fc = torch.nn.Linear(2, 1)\n",
    "criterion = torch.nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(fc.parameters())\n",
    "weights, bias = fc.parameters()\n",
    "\n",
    "for step in range(30001):\n",
    "    if step:\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    \n",
    "    pred = fc(x)\n",
    "    loss = criterion(pred, y)\n",
    "    \n",
    "    if step % 1000 == 0:\n",
    "        print('step = {}, loss = {:g}, weights = {}, bias={}'.format(\n",
    "                step, loss, weights[0, :].tolist(), bias.item()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "归一化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.tensor([[1, 1, 1], [2, 3, 1], [3, 5, 1],\n",
    "        [4, 2, 1], [5, 4, 1]], dtype=torch.float32)\n",
    "y = torch.tensor([-10, 12, 14, 16, 18],\n",
    "        dtype=torch.float32)\n",
    "\n",
    "x_mean, x_std = torch.mean(x, dim=0), torch.std(x, dim=0)\n",
    "x_mean[-1], x_std[-1] = 0, 1\n",
    "x_norm = (x - x_mean) / x_std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "无归一化和有归一化的比较"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step = 0, loss = 1.37277e+12\n",
      "step = 1000, loss = 5.97804e+06\n",
      "step = 2000, loss = 436378\n",
      "step = 3000, loss = 436378\n",
      "step = 4000, loss = 436377\n",
      "step = 5000, loss = 436376\n",
      "step = 6000, loss = 436375\n",
      "step = 7000, loss = 436372\n",
      "step = 8000, loss = 436368\n",
      "step = 9000, loss = 436361\n",
      "step = 10000, loss = 436349\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1000000, 0.0001], [2000000, 0.0003],\n",
    "        [3000000, 0.0005], [4000000, 0.0002], [5000000, 0.0004]])\n",
    "y = torch.tensor([-1000., 1200., 1400., 1600., 1800.]).reshape(-1, 1)\n",
    "\n",
    "fc = torch.nn.Linear(2, 1)\n",
    "criterion = torch.nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(fc.parameters())\n",
    "\n",
    "for step in range(10001):\n",
    "    if step:\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    pred = fc(x)\n",
    "    loss = criterion(pred, y)\n",
    "    if step % 1000 == 0:\n",
    "        print('step = {}, loss = {:g}'.format(step, loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step = 0, loss = 1.46595e+06\n",
      "step = 1000, loss = 243642\n",
      "step = 2000, loss = 213667\n",
      "step = 3000, loss = 213333\n",
      "step = 4000, loss = 213333\n",
      "step = 5000, loss = 213333\n",
      "step = 6000, loss = 213333\n",
      "step = 7000, loss = 213333\n",
      "step = 8000, loss = 213333\n",
      "step = 9000, loss = 213333\n",
      "step = 10000, loss = 213333\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([[1000000, 0.0001], [2000000, 0.0003],\n",
    "        [3000000, 0.0005], [4000000, 0.0002], [5000000, 0.0004]])\n",
    "y = torch.tensor([-1000., 1200., 1400., 1600., 1800.]).reshape(-1, 1)\n",
    "\n",
    "x_mean, x_std = torch.mean(x, dim=0), torch.std(x, dim=0)\n",
    "x_norm = (x - x_mean) / x_std\n",
    "y_mean, y_std = torch.mean(y, dim=0), torch.std(y, dim=0)\n",
    "y_norm = (y - y_mean) / y_std\n",
    "\n",
    "fc = torch.nn.Linear(2, 1)\n",
    "criterion = torch.nn.MSELoss()\n",
    "optimizer = torch.optim.Adam(fc.parameters())\n",
    "\n",
    "for step in range(10001):\n",
    "    if step:\n",
    "        optimizer.zero_grad()\n",
    "        loss_norm.backward()\n",
    "        optimizer.step()\n",
    "    pred_norm = fc(x_norm)\n",
    "    loss_norm = criterion(pred_norm, y_norm)\n",
    "    pred = pred_norm * y_std + y_mean\n",
    "    loss = criterion(pred, y)\n",
    "    if step % 1000 == 0:\n",
    "        print('step = {}, loss = {:g}'.format(step, loss))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
